﻿namespace PowerSong.Indexer
    {
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.IO;
    using System.Text;

    public class Indexer
        {

        // Key = File index, Value = File name
        private Dictionary<int, string> files = new Dictionary<int, string>();

        // Key = File name, Value = File index
        private Dictionary<string, int> fileNumbers = new Dictionary<string, int>();

        private int lastAddedFileIndex = 0;

        // Key = Word index, Value = Word
        private Dictionary<int, string> words = new Dictionary<int, string>();

        // Key = Word, Value = Word index
        private Dictionary<string, int> wordNumbers = new Dictionary<string, int>();

        private int lastAddedWordIndex = 0;

        // Key = Word index
        private Dictionary<int, WordOccurrences> index = new Dictionary<int, WordOccurrences>();

        /// <summary>
        /// Returns the complete list of document numbers.
        /// </summary>
        public int[] AllDocumentsNumbers
            {
            get
                {
                var result = new int[fileNumbers.Count];
                int i = 0;
                foreach (KeyValuePair<int, string> document in files)
                    {
                    result[i] = document.Key;
                    i += 1;
                    }
                return result;
                }
            }

        /// <summary>
        /// Returns the total number of documents indexed.
        /// </summary>
        public int DocumentCount
            {
            get { return files.Count; }
            }

        /// <summary>
        /// Returns the name of the document, given its index.
        /// </summary>
        public string GetDocumentName(int index)
            {
            if (files.ContainsKey(index))
                {
                return files[index];
                }
            else
                {
                throw new ApplicationException("Document " + index.ToString() + " does not exist in the index.");
                }
            }

        /// <summary>
        /// Returns the numeric index of the document, given its name.
        /// </summary>
        public int GetDocumentNumber(string name)
            {
            if (fileNumbers.ContainsKey(name))
                {
                return fileNumbers[name];
                }
            else
                {
                throw new ApplicationException("The document '" + name + "' does not exist in the index.");
                }
            }

        /// <summary>
        /// Returns the document indices of documents containing the specified words.
        /// </summary>
        /// <param name="word">The word to search for.</param>
        public List<int> DocumentsContainingWord(string word)
            {

            var result = new List<int>();
            int WordIndex = -1;

            if (wordNumbers.TryGetValue(word.ToUpper(), out WordIndex))
                {
                WordOccurrences occurrences = null;
                bool found = index.TryGetValue(WordIndex, out occurrences);

                if (!found)
                    {
                    throw new Exception("The word " + word + " exists in the words collection but not the index itself.");
                    }
                else
                    {
                    foreach (int documentIndex in index[WordIndex].Documents.Keys)
                        {
                        result.Add(documentIndex);
                        }
                    }

                }
            return result;

            }

        /// <summary>
        /// Returns the number of times the given word occurs in the given document.
        /// </summary>
        /// <param name="documentIndex">The document's index to search in.</param>
        /// <param name="word">The word to search for.</param>
        public int TermFrequency(int documentIndex, string word)
            {

            int result = 0;
            int wordIndex = -1;
            bool wordFound = wordNumbers.TryGetValue(word.ToUpper(), out wordIndex);

            // Get the document frequency and return it
            if (wordFound)
                {
                WordOccurrences occurrences = index[wordIndex];
                occurrences.Documents.TryGetValue(documentIndex, out result);
                }

            return result;

            }

        /// <summary>
        /// Empties the index.
        /// </summary>
        public void Clear()
            {
            files.Clear();
            fileNumbers.Clear();

            words.Clear();
            wordNumbers.Clear();

            lastAddedWordIndex = 0;
            lastAddedFileIndex = 0;

            index.Clear();
            }

        /// <summary>
        /// Returns the unique words in the given file's contents.
        /// </summary>
        /// <param name="fileContents">The textual contents of a file.</param>
        /// <returns>A list of the unique words in the given text.</returns>
        private string[] GetVocabulary(string fileContents)
            {

            fileContents = fileContents.ToUpper();
            string[] wordsInFile = SplitIntoFixedWords(fileContents);
            SortedList<string, int> vocabulary = new SortedList<string, int>();
            foreach (string word in wordsInFile)
                {
                if (!vocabulary.ContainsKey(word)) vocabulary.Add(word, 0);
                }
            string[] result = new string[vocabulary.Count];
            vocabulary.Keys.CopyTo(result, 0);
            return result;

            }

        /// <summary>
        /// Returns the number of times each word occurs in the given document.
        /// </summary>
        /// <param name="fileContents">The document containing words to count.</param>
        private Dictionary<int, int> GetWordCounts(string fileContents)
            {

            // Create a list of all the words, each set as occurring 0 times
            Dictionary<int, int> wordCounts = new Dictionary<int, int>();
            foreach (int index in words.Keys)
                {
                wordCounts.Add(index, 0);
                }

            // Count the number of occurrances of every word in this document
            string[] wordsInFile = SplitIntoFixedWords(fileContents);
            foreach (string word in wordsInFile)
                {
                int wordIndex = wordNumbers[word];
                wordCounts[wordIndex] += 1;
                }

            // Determine which words have a count greater than zero and include only those in the result
            List<int> wordsToInclude = new List<int>();
            foreach (int wordIndex in wordCounts.Keys)
                {
                if (wordCounts[wordIndex] > 0) wordsToInclude.Add(wordIndex);
                }
            var result = new Dictionary<int, int>();
            foreach (int wordIndex in wordsToInclude)
                {
                result.Add(wordIndex, wordCounts[wordIndex]);
                }

            return result;

            }

        /// <summary>
        /// Splits the document into words.
        /// </summary>
        /// <param name="fileContents">The document to split into words.</param>
        /// <returns>The document, split into words.</returns>
        private string[] SplitIntoFixedWords(string fileContents)
            {

            fileContents = fileContents.ToUpper();
            var splitters = new string[] { " ", "!", ".", ":", "\"", "-", Environment.NewLine };
            string[] words = fileContents.Split(splitters, StringSplitOptions.None);
            string[] wordsToReturn = new string[words.Length];
            int index = 0;
            foreach (string word in words)
                {
                var fixedWord = FixWord(word);
                if (!string.IsNullOrEmpty(fixedWord))
                    {
                    wordsToReturn[index] = fixedWord;
                    index += 1;
                    }
                }

            Array.Resize(ref wordsToReturn, index);
            return wordsToReturn;

            }

        /// <summary>
        /// Find each file in the given directory and index it.
        /// </summary>
        /// <param name="directory">The directory whose files to index.</param>
        /// <remarks>Files are not recursed, and so only the files in the specified directory itself will be indexed.</remarks>
        public void AddDirectoryToIndex(string directory)
            {

            // Get the file names
            foreach (string fileName in Directory.GetFiles(directory))
                {
                AddFileName(fileName);
                }

            // Phase 1: Build a global vocabulary
            SortedList<string, int> vocabulary = new SortedList<string, int>();
            foreach (KeyValuePair<int, string> file in files)
                {
                var contents = File.ReadAllText(file.Value);

                // Build a vocabulary for the file and add it to the complete list of words
                string[] fileVocabulary = GetVocabulary(contents);
                foreach (string word in fileVocabulary)
                    {
                    if (!vocabulary.ContainsKey(word)) vocabulary.Add(word, 0);
                    }

                }

            // Copy the complete list of words into the global vocabulary
            foreach (string word in vocabulary.Keys)
                {
                AddWordToVocabulary(word);
                }

            // Phase 2: Determine how many times each word occurs in each document
            foreach (KeyValuePair<int, string> file in files)
                {

                // Count each word as it is read in
                string contents = File.ReadAllText(file.Value);
                Dictionary<int, int> documentWordCount = GetWordCounts(contents);

                // Store number of times each word occurs in the document (by merging with the global index)
                MergeDocumentWordCount(fileNumbers[file.Value], documentWordCount);

                }

            }

        public void AddFileToIndex(string fileName)
            {
            AddFileContentsToIndex(fileName, File.ReadAllText(fileName));
            }

        public void AddFileToIndex(string virtualFilename, string fileName)
            {
            AddFileContentsToIndex(virtualFilename, File.ReadAllText(fileName));
            }


        public void AddFileContentsToIndex(string virtualFilename, string contents)
            {

            // Add to the list of files
            int fileIndex = AddFileName(virtualFilename);

            // Build a vocabulary for the file and add it to the complete list of words
            var vocabulary = new SortedList<string, int>();
            foreach (string word in GetVocabulary(contents))
                {
                if (!vocabulary.ContainsKey(word)) vocabulary.Add(word, 0);
                }

            // Copy the complete list of words into the global vocabulary
            foreach (string word in vocabulary.Keys)
                {
                if (!wordNumbers.ContainsKey(word)) AddWordToVocabulary(word);
                }

            // Count each word as it is read in
            Dictionary<int, int> documentWordCount = GetWordCounts(contents);

            // Store number of times each word occurs in the document (by merging with the global index)
            MergeDocumentWordCount(fileIndex, documentWordCount);

            }


        public void RemoveDocumentFromIndex(string virtualFilename)
            {

            // Check the file exists
            if (!fileNumbers.ContainsKey(virtualFilename))
                {
                throw new ApplicationException("The document '" + virtualFilename + "' does not exist in the index.");
                }

            // Remove the file from the list of files
            int documentIndex = fileNumbers[virtualFilename];
            files.Remove(documentIndex);
            fileNumbers.Remove(virtualFilename);

            // Process each word in the index
            var wordsToRemove = new List<int>();
            foreach (KeyValuePair<int, WordOccurrences> Word in index)
                {
                int wordIndex = Word.Key;
                WordOccurrences occurrences = Word.Value;

                // Remove the word from the index
                if (occurrences.Documents.ContainsKey(documentIndex))
                    {

                    // Remove the document from the word
                    int occurrencesInDocument = occurrences.Documents[documentIndex];
                    occurrences.Occurrences -= occurrencesInDocument;
                    occurrences.Documents.Remove(documentIndex);

                    // This word no longer exists in any document
                    if (occurrences.Documents.Count == 0) wordsToRemove.Add(wordIndex);

                    }

                }

            // Remove WordOccurrences objects that are no longer needed, as well as unused words in vocabulary
            foreach (int wordIndex in wordsToRemove)
                {
                index.Remove(wordIndex);
                wordNumbers.Remove(words[wordIndex]);
                words.Remove(wordIndex);
                }

            }

        /// <summary>
        /// Strips the words of all non-alphanumeric characters.
        /// </summary>
        /// <param name="word">The word to 'fix'.</param>
        /// <returns>The word without any symbol characters.</returns>
        private string FixWord(string word)
            {

            string result = string.Empty;
            foreach (char c in word)
                {
                if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'))
                    {
                    result += c;
                    }
                }
            return result;

            }

        /// <summary>
        /// Merges the document words count into the global index.
        /// </summary>
        /// <param name="documentFilenameIndex">The index of the document whose word counts to merge.</param>
        /// <param name="documentWordCount">A list of word indices and word counts.</param>
        private void MergeDocumentWordCount(int documentFilenameIndex, Dictionary<int, int> documentWordCount)
            {

            foreach (int wordIndex in documentWordCount.Keys)
                {
                int wordOccurrences = documentWordCount[wordIndex];
                if (index.ContainsKey(wordIndex))
                    {
                    var o = index[wordIndex];
                    o.Occurrences += wordOccurrences;
                    o.Documents.Add(documentFilenameIndex, wordOccurrences);
                    }
                else
                    {
                    index.Add(wordIndex, new WordOccurrences(wordOccurrences, documentFilenameIndex));
                    }
                }

            }

        private int AddWordToVocabulary(string word)
            {

            // Determine the next word index to use
            int wordIndex = lastAddedWordIndex;
            while (words.ContainsKey(wordIndex))
                {
                wordIndex += 1;
                }
            lastAddedWordIndex = wordIndex;

            // Add the word to the vocabulary
            words.Add(wordIndex, word);
            wordNumbers.Add(word, wordIndex);

            // Return the index
            return wordIndex;

            }

        private int AddFileName(string fileName)
            {

            // Determine the next file name index to use
            int fileNameIndex = lastAddedFileIndex;
            while (files.ContainsKey(fileNameIndex))
                {
                fileNameIndex += 1;
                }
            lastAddedFileIndex = fileNameIndex;

            // Add the word to the vocabulary
            files.Add(fileNameIndex, fileName);
            fileNumbers.Add(fileName, fileNameIndex);

            // Return the index
            return fileNameIndex;

            }

        public static Indexer LoadIndex(string fileName)
            {

            Indexer result = new Indexer();

            // Get list of files
            using (var file = new BinaryReader(new FileStream(fileName, FileMode.Open)))
                {

                int fileCount = file.ReadInt32();
                result.files.Clear();
                result.fileNumbers.Clear();
                for (int i = 1; i <= fileCount; i++)
                    {
                    int documentIndex = file.ReadInt32();
                    string documentName = file.ReadString();
                    result.files.Add(documentIndex, documentName);
                    result.fileNumbers.Add(documentName, documentIndex);
                    }

                // Get global vocabulary
                int vocabularyWordCount = file.ReadInt32();
                result.words.Clear();
                result.wordNumbers.Clear();
                for (int i = 1; i <= vocabularyWordCount; i++)
                    {
                    int wordIndex = file.ReadInt32();
                    string word = file.ReadString();
                    result.words.Add(wordIndex, word);
                    result.wordNumbers.Add(word, wordIndex);
                    }

                // Get the main part of the index
                int indexEntriesCount = file.ReadInt32();
                result.index.Clear();
                for (int a = 1; a <= indexEntriesCount; a++)
                    {
                    int mainKey = file.ReadInt32();
                    int occurrences = file.ReadInt32();
                    int documentCount = file.ReadInt32();
                    int[] documents = new int[documentCount];
                    for (int b = 0; b <= documentCount - 1; b++)
                        {
                        documents[b] = file.ReadInt32();
                        }
                    result.index.Add(mainKey, new WordOccurrences(occurrences, documents));
                    }

                // Close the file
                file.Close();

                }

            return result;

            }


        public void Save(string fileName)
            {

            // Write list of files
            BinaryWriter file = new BinaryWriter(new FileStream(fileName, FileMode.Create));
            file.Write(files.Count);
            foreach (KeyValuePair<int, string> indexedFilename in files)
                {
                file.Write(indexedFilename.Key);
                file.Write(indexedFilename.Value);
                }

            // Write global vocabulary
            file.Write(words.Count);
            foreach (KeyValuePair<int, string> word in words)
                {
                file.Write(word.Key);
                file.Write(word.Value);
                }

            // Write the main part of the index
            file.Write(index.Count);
            foreach (KeyValuePair<int, WordOccurrences> indexEntry in index)
                {
                file.Write(indexEntry.Key);
                file.Write(indexEntry.Value.Occurrences);
                file.Write(indexEntry.Value.Documents.Count);
                foreach (KeyValuePair<int, int> documentEntry in indexEntry.Value.Documents)
                    {
                    file.Write(documentEntry.Key);
                    }
                }

            // Close the file
            file.Close();

            }

        /// <summary>
        /// Returns a string describing the index in full.
        /// </summary>
        /// <remarks>
        /// The index is returned in the following format:
        /// 
        /// WORD: TotalOccurrances {DocumentFilename|DocumentOccurrances,...}
        /// </remarks>
        public override string ToString()
            {

            // Write out every word...
            var result = new StringBuilder();
            foreach (int wordIndex in index.Keys)
                {
                var o = index[wordIndex];
                result.Append(words[wordIndex]).Append(": ").Append(o.Occurrences).Append(" {");

                // ...and every occurrance of the word...
                bool commaRequired = false;
                foreach (int documentIndex in o.Documents.Keys)
                    {
                    string documentFilename = files[documentIndex];
                    int documentOccurrances = o.Documents[documentIndex];
                    if (commaRequired)
                        {
                        result.Append(",");
                        }
                    else
                        {
                        commaRequired = true;
                        }
                    result.Append(documentFilename).Append("|").Append(documentOccurrances.ToString());
                    }
                result.Append("}").Append(Environment.NewLine);
                }
            return result.ToString();

            }

        }

    }